#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Time    : 2024/9/11 20:16
# @Author  : 王凯
# @File    : main_run.py
# @Project : scrapy_spider
import sys
from pathlib import Path

from scrapy.crawler import CrawlerProcess
from scrapy.utils.project import get_project_settings

from apps.data_stats.data_stats.clean.gy_clean import GYClean
from apps.data_stats.data_stats.clean.pmi_clean import PmiClean

sys.path.append(Path(__file__).parent.parent.parent.parent.as_posix())
from components.settings.private.wfq_source_mysql_settings import *
from apps.data_stats.data_stats.clean.cl_clean import CLClean
from apps.data_stats.data_stats.clean.gdzc_claen import GdzcClaen
from apps.data_stats.data_stats.clean.hycw_clean import HycwClean
from utils.logs import logger

if __name__ == '__main__':
    settings = get_project_settings()
    os.environ['MYSQL_IP'] = MYSQL_IP
    os.environ['MYSQL_PORT'] = MYSQL_PORT
    os.environ['MYSQL_DB'] = MYSQL_DB
    os.environ['MYSQL_USER_NAME'] = MYSQL_USER_NAME
    os.environ['MYSQL_USER_PASS'] = MYSQL_USER_PASS
    process = CrawlerProcess(settings)
    spider_list = process.spider_loader.list()
    for spider in spider_list:
        process.crawl(spider)
    process.start()

    logger.info("clean task start")
    CLClean().run()
    GdzcClaen().run()
    HycwClean().run()
    PmiClean().run()
    GYClean().run()
